# Copyright (c) HySoP 2011-2024
#
# This file is part of HySoP software.
# See "https://particle_methods.gricad-pages.univ-grenoble-alpes.fr/hysop-doc/"
# for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Tools to collect time profiling information
for hysop classes.
"""
from hysop.core.mpi import main_rank
from hysop.tools.numpywrappers import npw
from hysop.tools.units import time2str
import numpy as np
[docs]
class FProfiler:
"""
Class for time measurments on the fly.
The objects can be linked to a class method.
"""
def __init__(self, fname):
"""Object to profile on the fly sections of code, methods ...
Usage:
>>> from hysop.tools.profiler import FProfiler
>>> from hysop.core.mpi import Wtime as ftime
>>> prof = FProfiler('some_name')
>>> start = ftime()
>>> # do something ... [1]
>>> prof += ftime() - start
>>> # do something else ... [2]
>>> prof += ftime() - start
>>> # ...
>>> # print(prof)
>>> # --> display total time spent in do [1] and [2]
>>> # and number of calls of prof
"""
# Function name
self.fname = fname
# Total execution time
self.total_time = 0.0
# Number of calls of the fprofiler
self.nb_calls = 0
[docs]
def get_name(self):
"""Profiler name"""
return self.fname
[docs]
def __iadd__(self, t):
"""+= operator"""
self.total_time += t
self.nb_calls += 1
return self
def __str__(self):
if self.nb_calls > 0:
s = "{} ncalls={}, total={}, mean={}".format(
self.fname,
self.nb_calls,
time2str(self.total_time),
time2str(self.total_time / self.nb_calls),
)
else:
s = ""
return s
[docs]
def get(self):
if self.nb_calls > 0:
return (
self.fname,
self.nb_calls,
time2str(self.total_time),
time2str(self.total_time / self.nb_calls),
)
else:
return ("", 0, None, None)
[docs]
class Profiler:
"""
Object used to collect profiling information inside operators.
"""
def __init__(self, obj):
"""
Collect profiling information for all operator
method decorate with @profile.
Parameters
----------
obj : object (python class) instance. See requirements in notes below.
Notes:
* obj must have '_get_profiling_info' and 'name'
attribute/method.
"""
self.summary = {}
self.table = []
# profiled object
self._obj = obj
_comm = self.get_comm()
_comm_size = _comm.Get_size()
# A dictionnary of profiled functions/methods as keys
# and elapsed time as value.
self._elems = {}
self._l = 1
self.all_data = None
self.node_id = None
[docs]
def down(self, l):
self._l = l + 1
[docs]
def get_name(self):
"""Return the name of the profiled object"""
_name = self._obj.name
return _name
[docs]
def get_comm(self):
"""Return the communicator associated to the profiled object"""
_comm = self._obj.mpi_params.comm
return _comm
[docs]
def __iadd__(self, other):
"""+= operator. Append a new profiled function to the collection"""
if not other.get_name() in self._elems.keys():
self._elems[str(other._obj)] = other
return self
def __setitem__(self, key, value):
self._elems[key] = value
def __getitem__(self, item):
try:
return self._elems[item]
except KeyError:
self._elems[item] = FProfiler(item)
return self._elems[item]
def __str__(self):
summary = self.summary
if len(summary) > 0:
if (
(self._l > 1)
and (len(summary) == 1)
and isinstance(next(iter(summary.values())), FProfiler)
):
s = f">{self.get_name()}::{next(iter(summary.values()))}"
else:
s = "{}[{}]>{}{}".format(
"\n" if (self._l == 1) else "",
main_rank,
self.get_name(),
" profiler report" if (self._l == 1) else "",
)
for (
v
) in (
summary.values()
): # sorted(summary.values(), key=lambda x: x.total_time):
if len(str(v)) > 0:
s += "\n{}".format(" " * self._l + str(v))
else:
s = ""
return s
[docs]
def write(self, prefix="", hprefix="", with_head=True):
"""
Parameters
----------
prefix : string, optional
hprefix : string, optional
with_head : bool, optional
"""
if prefix != "" and prefix[-1] != " ":
prefix += " "
if hprefix != "" and hprefix[-1] != " ":
hprefix += " "
if self._comm.Get_rank() == 0:
s = ""
h = hprefix + "Rank"
for r in range(self._comm_size):
s += prefix + f"{r}"
for i in range(len(self.all_names)):
s += f" {self.all_times[i][r]}"
s += "\n"
s += prefix + "-1"
for i in range(len(self.all_names)):
h += " " + self.all_names[i]
s += f" {self.all_times[i][self._comm_size]}"
h += "\n"
if with_head:
s = h + s
print(s)
[docs]
def summarize(self):
"""
Update profiling values and prepare data for a report
with print or write.
"""
# reset summary
self.summary = {}
# collect profiling results from decorated object(s), if any.
self._obj._get_profiling_info()
from hysop.fields.continuous_field import Field
i = 0
# Recursive summarize
for k in self._elems.keys():
try:
# Either elem[k] is a FProfiler ...
self.summary[self._elems[k].total_time] = self._elems[k]
except AttributeError:
# ... or a Profiler
i += 1
self._elems[k].down(self._l)
self._elems[k].summarize()
if isinstance(self._elems[k]._obj, Field):
self.summary[1e10 * i] = self._elems[k]
else:
self.summary[1e8 * i] = self._elems[k]
# Flatten elements
self.table = []
for k in sorted(self._elems.keys()):
if isinstance(self._elems[k], FProfiler):
tt = (
str(self._obj) + "." + k,
self._elems[k].total_time,
self._elems[k].nb_calls,
)
self.table.append(tt)
for k in sorted(self._elems.keys()):
if isinstance(self._elems[k], Profiler):
for e in self._elems[k].table:
tt = (str(self._obj) + "." + e[0], e[1], e[2])
self.table.append(tt)
[docs]
def tasks_summarize(self):
"""Collect all profiling data across processes."""
# Table content is (name, time, ncalls)
if self._l == 1:
all_data = self.all_data
comm = self.get_comm()
rk = comm.Get_rank()
comm_size = comm.Get_size()
nb = comm.allgather(len(self.table))
all_names = comm.allgather([_[0] for _ in self.table])
all_times = comm.allgather([_[1] for _ in self.table])
all_calls = comm.allgather([int(_[2]) for _ in self.table])
all_data = {}
# all_data structure : task_size, calls nb, self total, task mean, task min, task max
nelem = 6
for r in range(comm_size):
for n, t, c in zip(all_names[r], all_times[r], all_calls[r]):
if n not in all_data:
all_data[n] = [
0,
] * nelem + [
[],
]
all_data[n][0] += 1
all_data[n][1] += c
if r == rk:
all_data[n][2] = t
all_data[n][nelem].append(t)
for n in all_data.keys():
all_data[n][1] //= all_data[n][0]
all_data[n][3] = npw.average(all_data[n][nelem])
all_data[n][4] = npw.min(all_data[n][nelem])
all_data[n][5] = npw.max(all_data[n][nelem])
self.all_data = all_data